<?php


namespace app\modules\content\controllers;


use app\libs\Method;
use app\libs\QueryList;
use app\modules\content\models\Information;
use yii\web\Controller;

class SpiderController extends Controller
{
    public $safeFiled = ['name','keywords',"summary",'pic','content','viewCount','pushTime','belong'];

    /**
     * Notes:自动爬取脚本
     * User: monster
     * DateTime: 2021/11/12 17:41
     */
    public function actionAuto(){
        set_time_limit(0);
        $lists = \Yii::$app->params['spider_lists'];
        $lists = array_column($lists,null,'belong');
        foreach($lists as $key=>$value){
            $num=1;
            while ($this->SpiderInsert($value,$num)&&$num<=2){
                $num++;
            }
        }
        return Method::jsonGenerate(1,"","爬取完成");
    }


    public function SpiderInsert($dq_data,$page_index=1){
        $summary  =["",""];
        $keywords =["",""];
        $pic      =["",""];
        $model    ="";
        $charSet  ="utf-8";
        $removeHead = true;
        $g_url = $dq_data['url'];
        switch ($dq_data['type']){
            case 1:
                $name     =["dd>h2>a",'text'];
                $url      =["dd>h2>a",'href'];
                $summary  =["dd>p",'text',"-a"];
                $keywords =[".time_t a","text"];
                $pic      =["dt>a>img",'src'];
                $model    ="li>dl.fc";
                $g_url = $page_index==1?$dq_data['url']:$dq_data['url']."?page=".$page_index;
                break;
            case 2 :
                $name     =["a","text"];
                $url      =["a",'href'];
                $model    =".ielts_news ul>li";
                $g_url = $page_index==1?$dq_data['url']:$this->getShanghai($dq_data,$page_index);
                break;
            case 3 :
                $name     =[".pull-right>p>a",'text'];
                $url      =[".pull-right>p>a",'href'];
                $summary  =[".pull-right>.text",'text'];
                $keywords =[".pull-right>dl>dt a","text"];
                $pic      =["div.pull-left>a>img",'src'];
                $model    =".those_content>.things_list";
                $g_url = $page_index==1?$dq_data['url']:$dq_data['url']."liebiao".$page_index.".html";
                break;
            case 4 :
                $name     =[".right_des>p>a",'text'];
                $url      =[".right_des>p>a",'href'];
                $summary  =[".right_des>span a",'text'];
                $pic      =[".left_img>a>img",'src'];
                $model    =".all_item_box>.item_box";
                $removeHead = false;
                $g_url = $page_index==1?$dq_data['url']:$dq_data['url']."?pn=".$page_index;
                break;
            case 5 :
                $name     =["dl>dt>a",'text'];
                $url      =["dl>dt>a",'href'];
                $summary  =["dl>dd",'text'];
                $keywords =["p","text"];
                $model    =".liebiao-div>.list-item";
                $g_url = $page_index==1?$dq_data['url']:$this->getTianDao($dq_data,$page_index);
                break;
            case 6 :
                $name     =[".indetail_list_right>.indetail_title",'text'];
                $url      =["",'href'];
                $summary  =[".indetail_list_right>.indetail_des",'text'];
                $model    =".indetail_container>.indetail_list";
                $g_url = $page_index==1?$dq_data['url']:$dq_data['url']."index_".$page_index.".html";
                break;
            case 7 :
                $name     =["em+a",'text'];
                $url      =["em+a",'href'];
                $keywords =["em>a","text"];
                $model    ="#li_list>li";
                $g_url = $page_index==1?$dq_data['url']:$this->getXDF($dq_data,$page_index);
                break;
        }

            $data =  QueryList::Query($g_url,[
                'name'          =>  $name,
                'url'         =>  $url,
                'keywords'  =>  $keywords,
                'summary'  =>  $summary,
                'pic'  =>  $pic,
            ],$model,'UTF-8',$charSet,$removeHead)->data;
            foreach ($data as $key => $value){
                if (!empty(Information::find()->asArray()->where(['like','name',$value['name']])->all())){ //不为空就在data删除这个数据
                    unset($data[$key]);
                }else{
                    $data[$key]['pic'] = $this->handPic($dq_data,$value['pic']);
                    $data[$key]['content'] = $this->getContent($dq_data,$this->getContentUrl($dq_data,$value['url'])); //封装content进入data 批量插入
                    unset($data[$key]['url']);
                    $data[$key]['viewCount'] = rand(200,500);
                    $data[$key]['pushTime'] = time();
                    $data[$key]['belong'] = $dq_data['belong'];
                }
            }
            if (empty($data))
                return false; //说明这个网址没有新数据
            \Yii::$app->db->createCommand()->batchInsert(Information::tableName(),$this->safeFiled,$data)->execute(); //批量插入
            return true;
    }


    public function getContent($dq_data,$url){
        $charSet = "utf-8";
        $removeHead = true;
        switch ($dq_data['type']){
            case 1:
                $content = ["","html"];
                $model = ".information-text-box";
                break;
            case 2:
                $content = [".zw","html"];
                $model = ".ielts_con";
                break;
            case 3:
//                $content = ["","html","-h1 -dl -.freeDataGet -.first-words -p:eq(-1) -p:eq(-2) -p:eq(-3) -p:eq(-4) -p:eq(-5) -p:eq(-6) -.page"];
                $content = ["","html"];
                $model = ".content-left>.article-content.clearfix";
                break;
            case 4:
                $content = [".articel_main","html","-h1 -.time_edit -.des -.share_box -.tager_box"];
                $model = ".left_articel";
                $removeHead = false;
                break;
            case 5:
                $content = [".passage","html"];
                $model = ".passage-div";
                break;
            case 6:
                $content = [".indetail_sd_content","html"];
                $model = ".indetail_container";
                break;
            case 7:
                $content = ["","html","-div:eq(0)"];
                $model = ".air_con";
                break;
        }
        $data = QueryList::Query($url,array(
            'content' => $content
        ),$model,'UTF-8',$charSet,$removeHead)->data;
        if (empty($data[0]['content'])) return "";
        return $data[0]['content'];
    }

    /**
     * Notes:上海环球雅思 url
     * User: monster
     * DateTime: 2021/11/12 10:44
     * @param $dq_data
     * @param $page
     * @return string url
     */
    public function getShanghai($dq_data,$page){
        $url = $dq_data['url'];
        switch ($dq_data['belong']){
            case 2:
                $url = $dq_data['url']."list_51_".$page.".html";
                break;
            case 3:
                $url = $dq_data['url']."list_52_".$page.".html";
                break;
            case 4:
                $url = $dq_data['url']."list_53_".$page.".html";
                break;
            case 5:
                $url = $dq_data['url']."list_50_".$page.".html";
                break;
            case 6:
                $url = $dq_data['url']."list_54_".$page.".html";
                break;
            case 7:
                $url = $dq_data['url']."list_55_".$page.".html";
                break;
            case 8:
                $url = $dq_data['url']."list_82_".$page.".html";
                break;
            case 9:
                $url = $dq_data['url']."list_78_".$page.".html";
                break;
        }
        return $url;
    }

    /**
     * Notes: 得到天道教育分页url
     * User: monster
     * DateTime: 2021/11/12 10:57
     * @param $dq_data
     * @param $page
     * @return string url
     */
    public function getTianDao($dq_data,$page){
        $url = $dq_data['url'];
        switch ($dq_data['belong']){
            case 23:
                $url = $dq_data['url']."list_137_".$page.".html";
                break;
            case 24:
                $url = $dq_data['url']."list_139_".$page.".html";
                break;
            case 25:
                $url = $dq_data['url']."list_138_".$page.".html";
                break;
            case 26:
                $url = $dq_data['url']."list_142_".$page.".html";
                break;
            case 27:
                $url = $dq_data['url']."list_143_".$page.".html";
                break;
            case 28:
                $url = $dq_data['url']."list_144_".$page.".html";
                break;
            case 29:
                $url = $dq_data['url']."list_140_".$page.".html";
                break;
            case 30:
                $url = $dq_data['url']."list_141_".$page.".html";
                break;
            case 31:
                $url = $dq_data['url']."list_147_".$page.".html";
                break;
            case 32:
                $url = $dq_data['url']."list_".$page.".html";
                break;
        }
        return $url;
    }


    /**
     * Notes:得到新东方 分页url
     * User: monster
     * DateTime: 2021/11/12 11:02
     * @param $dq_data
     * @param $page
     * @return string
     */
    public function getXDF($dq_data,$page){
        switch ($dq_data['belong']){
            case 42:
                $url = "http://ielts.xdf.cn/list_1063_".$page.".html";
                break;
            case 43:
                $url = "http://ielts.xdf.cn/list_1293_".$page.".html";
                break;
            case 44:
                $url = "http://ielts.xdf.cn/list_1066_".$page.".html";
                break;
            case 45:
                $url = "http://ielts.xdf.cn/list_1067_".$page.".html";
                break;
            case 46:
                $url = "http://ielts.xdf.cn/list_1068_".$page.".html";
                break;
            case 47:
                $url = "http://ielts.xdf.cn/list_1070_".$page.".html";
                break;
            case 48:
                $url = "http://ielts.xdf.cn/list_1071_".$page.".html";
                break;
            case 49:
                $url = "http://ielts.xdf.cn/list_2694_".$page.".html";
                break;
        }
        return $url;
    }

    /**
     * Notes:拼接 图片网络地址
     * User: monster
     * DateTime: 2021/11/12 11:38
     * @param $dq_data
     * @param $url
     * @return string
     */
    public function handPic($dq_data,$url){
        $img_url = $url;
        switch ($dq_data['type']){
            case 1 :
            case 4:
                $img_url = $dq_data['domain'].$url;
                break;
        }
        return $img_url;
    }

    /**
     * Notes:拼接 详情的url
     * User: monster
     * DateTime: 2021/11/12 11:53
     * @param $dq_data
     * @param $url
     * @return string
     */
    public function getContentUrl($dq_data,$url){
        $content_url = $url;
        switch ($dq_data['type']){
            case 1 :
            case 2 :
            case 4 :
            case 6 :
                $content_url = $dq_data['domain'].$url;
                break;
            case 3 :
                $content_url = "https:".$url;
                break;
        }
        return $content_url;
    }

}